Find the right job for right jobs/problems
Supervised
Unsupervised
-yi *xi if yi * (W@x) < 1
Pros:
Cons:
Python is most popular / effective programming language for Machine Learning
Popular ML packages
## import libraries
import numpy as np
import pandas as pd
## data visualization
import seaborn as sns
import matplotlib.pyplot as plt
## algorithm spliting train and test data
from sklearn.model_selection import train_test_split
## metric algorithm
from sklearn import metrics
## SVM Python Programming
### Import data
from sklearn import datasets
iris = datasets.load_iris()
X = iris["data"] # sepal length, sepal width, petal length, petal width
y = iris["target"] # 'setosa', 'versicolor', 'virginica'
iris
{'data': array([[5.1, 3.5, 1.4, 0.2],
[4.9, 3. , 1.4, 0.2],
[4.7, 3.2, 1.3, 0.2],
[4.6, 3.1, 1.5, 0.2],
[5. , 3.6, 1.4, 0.2],
[5.4, 3.9, 1.7, 0.4],
[4.6, 3.4, 1.4, 0.3],
[5. , 3.4, 1.5, 0.2],
[4.4, 2.9, 1.4, 0.2],
[4.9, 3.1, 1.5, 0.1],
[5.4, 3.7, 1.5, 0.2],
[4.8, 3.4, 1.6, 0.2],
[4.8, 3. , 1.4, 0.1],
[4.3, 3. , 1.1, 0.1],
[5.8, 4. , 1.2, 0.2],
[5.7, 4.4, 1.5, 0.4],
[5.4, 3.9, 1.3, 0.4],
[5.1, 3.5, 1.4, 0.3],
[5.7, 3.8, 1.7, 0.3],
[5.1, 3.8, 1.5, 0.3],
[5.4, 3.4, 1.7, 0.2],
[5.1, 3.7, 1.5, 0.4],
[4.6, 3.6, 1. , 0.2],
[5.1, 3.3, 1.7, 0.5],
[4.8, 3.4, 1.9, 0.2],
[5. , 3. , 1.6, 0.2],
[5. , 3.4, 1.6, 0.4],
[5.2, 3.5, 1.5, 0.2],
[5.2, 3.4, 1.4, 0.2],
[4.7, 3.2, 1.6, 0.2],
[4.8, 3.1, 1.6, 0.2],
[5.4, 3.4, 1.5, 0.4],
[5.2, 4.1, 1.5, 0.1],
[5.5, 4.2, 1.4, 0.2],
[4.9, 3.1, 1.5, 0.2],
[5. , 3.2, 1.2, 0.2],
[5.5, 3.5, 1.3, 0.2],
[4.9, 3.6, 1.4, 0.1],
[4.4, 3. , 1.3, 0.2],
[5.1, 3.4, 1.5, 0.2],
[5. , 3.5, 1.3, 0.3],
[4.5, 2.3, 1.3, 0.3],
[4.4, 3.2, 1.3, 0.2],
[5. , 3.5, 1.6, 0.6],
[5.1, 3.8, 1.9, 0.4],
[4.8, 3. , 1.4, 0.3],
[5.1, 3.8, 1.6, 0.2],
[4.6, 3.2, 1.4, 0.2],
[5.3, 3.7, 1.5, 0.2],
[5. , 3.3, 1.4, 0.2],
[7. , 3.2, 4.7, 1.4],
[6.4, 3.2, 4.5, 1.5],
[6.9, 3.1, 4.9, 1.5],
[5.5, 2.3, 4. , 1.3],
[6.5, 2.8, 4.6, 1.5],
[5.7, 2.8, 4.5, 1.3],
[6.3, 3.3, 4.7, 1.6],
[4.9, 2.4, 3.3, 1. ],
[6.6, 2.9, 4.6, 1.3],
[5.2, 2.7, 3.9, 1.4],
[5. , 2. , 3.5, 1. ],
[5.9, 3. , 4.2, 1.5],
[6. , 2.2, 4. , 1. ],
[6.1, 2.9, 4.7, 1.4],
[5.6, 2.9, 3.6, 1.3],
[6.7, 3.1, 4.4, 1.4],
[5.6, 3. , 4.5, 1.5],
[5.8, 2.7, 4.1, 1. ],
[6.2, 2.2, 4.5, 1.5],
[5.6, 2.5, 3.9, 1.1],
[5.9, 3.2, 4.8, 1.8],
[6.1, 2.8, 4. , 1.3],
[6.3, 2.5, 4.9, 1.5],
[6.1, 2.8, 4.7, 1.2],
[6.4, 2.9, 4.3, 1.3],
[6.6, 3. , 4.4, 1.4],
[6.8, 2.8, 4.8, 1.4],
[6.7, 3. , 5. , 1.7],
[6. , 2.9, 4.5, 1.5],
[5.7, 2.6, 3.5, 1. ],
[5.5, 2.4, 3.8, 1.1],
[5.5, 2.4, 3.7, 1. ],
[5.8, 2.7, 3.9, 1.2],
[6. , 2.7, 5.1, 1.6],
[5.4, 3. , 4.5, 1.5],
[6. , 3.4, 4.5, 1.6],
[6.7, 3.1, 4.7, 1.5],
[6.3, 2.3, 4.4, 1.3],
[5.6, 3. , 4.1, 1.3],
[5.5, 2.5, 4. , 1.3],
[5.5, 2.6, 4.4, 1.2],
[6.1, 3. , 4.6, 1.4],
[5.8, 2.6, 4. , 1.2],
[5. , 2.3, 3.3, 1. ],
[5.6, 2.7, 4.2, 1.3],
[5.7, 3. , 4.2, 1.2],
[5.7, 2.9, 4.2, 1.3],
[6.2, 2.9, 4.3, 1.3],
[5.1, 2.5, 3. , 1.1],
[5.7, 2.8, 4.1, 1.3],
[6.3, 3.3, 6. , 2.5],
[5.8, 2.7, 5.1, 1.9],
[7.1, 3. , 5.9, 2.1],
[6.3, 2.9, 5.6, 1.8],
[6.5, 3. , 5.8, 2.2],
[7.6, 3. , 6.6, 2.1],
[4.9, 2.5, 4.5, 1.7],
[7.3, 2.9, 6.3, 1.8],
[6.7, 2.5, 5.8, 1.8],
[7.2, 3.6, 6.1, 2.5],
[6.5, 3.2, 5.1, 2. ],
[6.4, 2.7, 5.3, 1.9],
[6.8, 3. , 5.5, 2.1],
[5.7, 2.5, 5. , 2. ],
[5.8, 2.8, 5.1, 2.4],
[6.4, 3.2, 5.3, 2.3],
[6.5, 3. , 5.5, 1.8],
[7.7, 3.8, 6.7, 2.2],
[7.7, 2.6, 6.9, 2.3],
[6. , 2.2, 5. , 1.5],
[6.9, 3.2, 5.7, 2.3],
[5.6, 2.8, 4.9, 2. ],
[7.7, 2.8, 6.7, 2. ],
[6.3, 2.7, 4.9, 1.8],
[6.7, 3.3, 5.7, 2.1],
[7.2, 3.2, 6. , 1.8],
[6.2, 2.8, 4.8, 1.8],
[6.1, 3. , 4.9, 1.8],
[6.4, 2.8, 5.6, 2.1],
[7.2, 3. , 5.8, 1.6],
[7.4, 2.8, 6.1, 1.9],
[7.9, 3.8, 6.4, 2. ],
[6.4, 2.8, 5.6, 2.2],
[6.3, 2.8, 5.1, 1.5],
[6.1, 2.6, 5.6, 1.4],
[7.7, 3. , 6.1, 2.3],
[6.3, 3.4, 5.6, 2.4],
[6.4, 3.1, 5.5, 1.8],
[6. , 3. , 4.8, 1.8],
[6.9, 3.1, 5.4, 2.1],
[6.7, 3.1, 5.6, 2.4],
[6.9, 3.1, 5.1, 2.3],
[5.8, 2.7, 5.1, 1.9],
[6.8, 3.2, 5.9, 2.3],
[6.7, 3.3, 5.7, 2.5],
[6.7, 3. , 5.2, 2.3],
[6.3, 2.5, 5. , 1.9],
[6.5, 3. , 5.2, 2. ],
[6.2, 3.4, 5.4, 2.3],
[5.9, 3. , 5.1, 1.8]]),
'target': array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]),
'frame': None,
'target_names': array(['setosa', 'versicolor', 'virginica'], dtype='<U10'),
'DESCR': '.. _iris_dataset:\n\nIris plants dataset\n--------------------\n\n**Data Set Characteristics:**\n\n :Number of Instances: 150 (50 in each of three classes)\n :Number of Attributes: 4 numeric, predictive attributes and the class\n :Attribute Information:\n - sepal length in cm\n - sepal width in cm\n - petal length in cm\n - petal width in cm\n - class:\n - Iris-Setosa\n - Iris-Versicolour\n - Iris-Virginica\n \n :Summary Statistics:\n\n ============== ==== ==== ======= ===== ====================\n Min Max Mean SD Class Correlation\n ============== ==== ==== ======= ===== ====================\n sepal length: 4.3 7.9 5.84 0.83 0.7826\n sepal width: 2.0 4.4 3.05 0.43 -0.4194\n petal length: 1.0 6.9 3.76 1.76 0.9490 (high!)\n petal width: 0.1 2.5 1.20 0.76 0.9565 (high!)\n ============== ==== ==== ======= ===== ====================\n\n :Missing Attribute Values: None\n :Class Distribution: 33.3% for each of 3 classes.\n :Creator: R.A. Fisher\n :Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov)\n :Date: July, 1988\n\nThe famous Iris database, first used by Sir R.A. Fisher. The dataset is taken\nfrom Fisher\'s paper. Note that it\'s the same as in R, but not as in the UCI\nMachine Learning Repository, which has two wrong data points.\n\nThis is perhaps the best known database to be found in the\npattern recognition literature. Fisher\'s paper is a classic in the field and\nis referenced frequently to this day. (See Duda & Hart, for example.) The\ndata set contains 3 classes of 50 instances each, where each class refers to a\ntype of iris plant. One class is linearly separable from the other 2; the\nlatter are NOT linearly separable from each other.\n\n.. topic:: References\n\n - Fisher, R.A. "The use of multiple measurements in taxonomic problems"\n Annual Eugenics, 7, Part II, 179-188 (1936); also in "Contributions to\n Mathematical Statistics" (John Wiley, NY, 1950).\n - Duda, R.O., & Hart, P.E. (1973) Pattern Classification and Scene Analysis.\n (Q327.D83) John Wiley & Sons. ISBN 0-471-22361-1. See page 218.\n - Dasarathy, B.V. (1980) "Nosing Around the Neighborhood: A New System\n Structure and Classification Rule for Recognition in Partially Exposed\n Environments". IEEE Transactions on Pattern Analysis and Machine\n Intelligence, Vol. PAMI-2, No. 1, 67-71.\n - Gates, G.W. (1972) "The Reduced Nearest Neighbor Rule". IEEE Transactions\n on Information Theory, May 1972, 431-433.\n - See also: 1988 MLC Proceedings, 54-64. Cheeseman et al"s AUTOCLASS II\n conceptual clustering system finds 3 classes in the data.\n - Many, many more ...',
'feature_names': ['sepal length (cm)',
'sepal width (cm)',
'petal length (cm)',
'petal width (cm)'],
'filename': 'C:\\Users\\kevin\\anaconda3\\envs\\python-training\\lib\\site-packages\\sklearn\\datasets\\data\\iris.csv'}
## Data Analysis
print("Shape of input data X :", X.shape)
print("Shape of result(label) y:",y.shape)
print(X[[0, 1, 50, 149], :])
print(y[[0, 1, 50, 149]])
Shape of input data X : (150, 4) Shape of result(label) y: (150,) [[5.1 3.5 1.4 0.2] [4.9 3. 1.4 0.2] [7. 3.2 4.7 1.4] [5.9 3. 5.1 1.8]] [0 0 1 2]
## Set up dataframe for IRIS data
df_iris = pd.DataFrame(iris['data'], columns=iris['feature_names'])
df_iris.head()
| sepal length (cm) | sepal width (cm) | petal length (cm) | petal width (cm) | |
|---|---|---|---|---|
| 0 | 5.1 | 3.5 | 1.4 | 0.2 |
| 1 | 4.9 | 3.0 | 1.4 | 0.2 |
| 2 | 4.7 | 3.2 | 1.3 | 0.2 |
| 3 | 4.6 | 3.1 | 1.5 | 0.2 |
| 4 | 5.0 | 3.6 | 1.4 | 0.2 |
## set up dataframe of input and target data
df_iris1 = pd.DataFrame(iris['data'], columns=iris['feature_names'])
df_iris2 = pd.DataFrame(iris['target'], columns=['target'])
print(df_iris1.head())
print(df_iris2.head())
sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 target 0 0 1 0 2 0 3 0 4 0
## Converting factor preparation
num_to_char = {
0: 'Setosa', 1:'Vesicolor', 2:'Virginica'
}
## joining by index number between
df_iris3 = df_iris1.join(df_iris2)
print(df_iris3.head())
## create new variable by converting target to species
df_iris3['species'] = df_iris3['target'].map(num_to_char)
df_iris3.head()
sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) \ 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 target 0 0 1 0 2 0 3 0 4 0
| sepal length (cm) | sepal width (cm) | petal length (cm) | petal width (cm) | target | species | |
|---|---|---|---|---|---|---|
| 0 | 5.1 | 3.5 | 1.4 | 0.2 | 0 | Setosa |
| 1 | 4.9 | 3.0 | 1.4 | 0.2 | 0 | Setosa |
| 2 | 4.7 | 3.2 | 1.3 | 0.2 | 0 | Setosa |
| 3 | 4.6 | 3.1 | 1.5 | 0.2 | 0 | Setosa |
| 4 | 5.0 | 3.6 | 1.4 | 0.2 | 0 | Setosa |
## Data Visualization
sns.set(style="ticks")
plt.figure(figsize=(12,10))
plt.subplot(2,2,1)
sns.boxplot(x='species',y='sepal length (cm)',data=df_iris3)
plt.subplot(2,2,2)
sns.boxplot(x='species',y='sepal width (cm)',data=df_iris3)
plt.subplot(2,2,3)
sns.boxplot(x='species',y='petal length (cm)',data=df_iris3)
plt.subplot(2,2,4)
sns.boxplot(x='species',y='petal width (cm)',data=df_iris3)
plt.show()
## subsetting data 25 setosa, 25 versicolor and 25 virginica
X2 = np.concatenate((X[:25], X[50:75], X[100:125]), axis=0)
y2 = np.concatenate((y[:25], y[50:75], y[100:125]))
X_train, X_test, y_train, y_test = train_test_split(X2, y2, test_size=0.2) ## splitting
## import modules
from sklearn.svm import SVC
## Pick up SVM Classifier model
svm_clf = SVC(kernel="linear", C=float("inf"))
## Train the model
svm_clf.fit(X_train, y_train)
SVC(C=inf, kernel='linear')
## Weights of Liner Kernel
print("Weights of Linear Kernel :\n", svm_clf.coef_)
print("\nBias of Linear Kernel :\n",svm_clf.intercept_)
Weights of Linear Kernel : [[ 0.10928963 0.49180334 -0.87431705 -0.27322408] [-0.07698231 0.07698231 -0.49268676 -0.23094692] [ 1.22211917 4.27532438 -3.51155404 -9.51592385]] Bias of Linear Kernel : [ 0.44262272 2.09160913 14.09290137]
## Predict using the trained SVM
svm_pred = svm_clf.predict((X_test))
print("Prection : ", svm_pred)
print("Test label: ", y_test)
Prection : [0 0 2 1 2 2 2 2 0 0 2 0 2 1 1] Test label: [0 0 2 1 2 2 2 2 0 0 2 0 2 1 1]
## Evaluation of trained SVM model
metrics.accuracy_score(svm_pred, y_test)
1.0
print("Linear coefficient of SVM :", svm_clf.coef_)
print("Linear Intercept of SVM :", svm_clf.intercept_)
Linear coefficient of SVM : [[ 0.10928963 0.49180334 -0.87431705 -0.27322408] [-0.07698231 0.07698231 -0.49268676 -0.23094692] [ 1.22211917 4.27532438 -3.51155404 -9.51592385]] Linear Intercept of SVM : [ 0.44262272 2.09160913 14.09290137]
How KNN works?
Pros:
Cons:
#import ML algorithm
from sklearn.neighbors import KNeighborsClassifier
#select and train model
#n_neighbors is the number of neighbor to use
knn = KNeighborsClassifier(n_neighbors=3)
knn.fit(X_train, y_train)
KNeighborsClassifier(n_neighbors=3)
#predict output
knn_pred = knn.predict(X_test)
print("Prection from KNN : ", knn_pred)
print("Test label: ", y_test)
Prection from KNN : [0 0 2 1 2 2 2 1 0 0 2 0 2 1 1] Test label: [0 0 2 1 2 2 2 2 0 0 2 0 2 1 1]
## Evaluation of trained SVM model
metrics.accuracy_score(knn_pred, y_test)
0.9333333333333333
The widely used machine learning algorithm that use tree to visually and explicitly represent decision and its conditions.
#import ML algorithm
from sklearn.tree import DecisionTreeClassifier
#select and train model
d_tree = DecisionTreeClassifier(max_depth=4)
d_tree.fit(X_train, y_train)
DecisionTreeClassifier(max_depth=4)
#predict output
dtree_pred = d_tree.predict(X_test)
print("Prediction from Decision Tree : ", dtree_pred)
print("Test label: ", y_test)
Prediction from Decision Tree : [0 0 2 1 2 2 2 1 0 0 2 0 2 1 1] Test label: [0 0 2 1 2 2 2 2 0 0 2 0 2 1 1]
## Evaluation of trained SVM model
metrics.accuracy_score(dtree_pred, y_test)
0.9333333333333333
The ensemble of decision trees that uses multiple decision trees.
Steps
Why Random Forest? Can build more robust model that has a better generalization performance and less subject to overfitting.
#import ML algorithm
from sklearn.ensemble import RandomForestClassifier
#select and train model (500 trees)
rdf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=15, n_jobs=1)
rdf.fit(X_train, y_train)
RandomForestClassifier(max_leaf_nodes=15, n_estimators=500, n_jobs=1)
#predict output
rdf_pred = rdf.predict(X_test)
print("Prediction from Random Forest : ", rdf_pred)
print("Test label: ", y_test)
Prediction from Random Forest : [0 0 2 1 2 2 2 1 0 0 2 0 2 1 1] Test label: [0 0 2 1 2 2 2 2 0 0 2 0 2 1 1]
## Evaluation of trained SVM model
metrics.accuracy_score(rdf_pred, y_test)
0.9333333333333333
Optimized Gradient Boostig tree-based algorithm that
Note : Random Forest model is a bagging ensemble model while XGBoost is a sequential ensemble model.
#import ML algorithm
import xgboost as xgb
#select and train model (500 trees)
#xgb_model = xgb.XGBClassifier(param, num_round=20)
xgb_model = xgb.XGBClassifier()
xgb_model.fit(X_train, y_train)
[23:04:35] WARNING: ..\src\learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
C:\Users\kevin\anaconda3\envs\python-training\lib\site-packages\xgboost\sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1]. warnings.warn(label_encoder_deprecation_msg, UserWarning)
XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.300000012, max_delta_step=0, max_depth=6,
min_child_weight=1, missing=nan, monotone_constraints='()',
n_estimators=100, n_jobs=12, num_parallel_tree=1,
objective='multi:softprob', random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=None, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None)
#predict output
xgb_model_pred = xgb_model.predict(X_test)
print("Prediction from Random Forest : ", xgb_model_pred)
print("Test label: ", y_test)
Prediction from Random Forest : [0 0 2 1 2 2 2 1 0 0 2 0 2 1 1] Test label: [0 0 2 1 2 2 2 2 0 0 2 0 2 1 1]
## Evaluation of trained SVM model
metrics.accuracy_score(xgb_model_pred, y_test)
0.9333333333333333
Hypothesis function : hθ(x) = 1 / (1 + e-z) where z= ( θ0x0 + θ1x1 + θ2x2 + θ3x3 +,, + θnxn )
## Prepare binary data
X_binary = X[y !=2]
y_binary = y[y !=2]
## Splitt 80 by 20
X_train_binary, X_test_binary, y_train_binary, y_test_binary = train_test_split(X_binary, y_binary, test_size=0.2)
print(y_train_binary)
print(np.unique(y_train_binary, return_counts=True))
[0 0 1 0 0 0 1 0 0 1 1 0 1 1 1 0 0 0 0 0 0 0 1 1 0 0 1 1 1 1 1 1 1 1 0 1 1 0 1 1 1 1 0 1 1 0 1 0 0 0 1 1 0 1 0 0 1 1 1 0 1 0 0 1 1 0 0 1 0 1 0 1 0 0 0 0 1 0 1 1] (array([0, 1]), array([39, 41], dtype=int64))
#import ML algorithm
from sklearn.linear_model import LogisticRegression
#select and train model
lr = LogisticRegression()
lr.fit(X_train_binary, y_train_binary)
LogisticRegression()
#predict output
lr_pred = lr.predict(X_test_binary)
print("Prediction from Random Forest : ", lr_pred)
print("Test label: ", y_test_binary)
Prediction from Random Forest : [1 1 0 0 1 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0] Test label: [1 1 0 0 1 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0]
## Evaluation of trained model
metrics.accuracy_score(lr_pred, y_test_binary)
1.0
## Coefficient
print("Weights of Logistic Regression :", lr.coef_)
print("Intercept of Logistic Regression :",lr.intercept_)
Weights of Logistic Regression : [[ 0.46732726 -0.85712984 2.17282351 0.91872566]] Intercept of Logistic Regression : [-6.46968917]

## Preparing Lineal Regression data
X_linear = 2 * np.random.rand(100, 1)
y_linear = 4 + 3 * X_linear + np.random.randn(100, 1)
## Visualization of data
import matplotlib.pyplot as plt
%matplotlib inline
plt.scatter(X_linear, y_linear)
<matplotlib.collections.PathCollection at 0x1ef439b6580>
## train and test data
X_train_linear, X_test_linear, y_train_linear, y_test_linear = train_test_split(X_linear, y_linear, test_size=0.2, random_state=42)
#import ML algorithm
from sklearn import linear_model
from sklearn.metrics import mean_squared_error
#select and train model
linear = linear_model.LinearRegression()
linear.fit(X_train_linear, y_train_linear)
LinearRegression()
#predict output
linear_pred = linear.predict(X_test_linear)
print("Prediction from Linear Regression : ", linear_pred.flatten())
print("Test label: ", y_test_linear.flatten())
Prediction from Linear Regression : [9.84973665 8.58049131 9.26180038 9.19728843 9.32556737 4.73004489 9.49702509 6.54572126 7.3342988 8.30844705 7.52538351 7.46020112 4.71869713 7.68526911 5.93799553 5.27515035 8.89894511 9.63326395 4.41427016 5.53627554] Test label: [ 7.62245279 7.28616881 8.38079342 7.87279706 9.54695661 4.42926622 10.93877677 7.95747427 8.07720866 9.17689425 7.83621229 6.59282446 5.57154181 7.96862209 5.67948783 6.41744524 8.63057114 11.09420594 4.88821005 5.66635402]
## Evaluation of trained SVM model
mean_squared_error(linear_pred, y_test_binary)
53.06637624404051
## Coefficient
print("Weights of Linear Regression :", linear.coef_)
print("Intercept of Linear Regression :", linear.intercept_)
Weights of Linear Regression : [[2.78281885]] Intercept of Linear Regression : [4.32763581]
How it works
#import ML algorithm
from sklearn.cluster import KMeans
# build k means model with 2 clusters
kmeans = KMeans(n_clusters = 3, tol=0.0001, max_iter=100000)
kmeans.fit(X)
KMeans(max_iter=100000, n_clusters=3)
#print the results of the value of centroids
print(kmeans.cluster_centers_) # centroids
#print the results of the value of centroids
print("prediction from Kmeans: \n", kmeans.labels_)
print("labels: \n", y)
[[5.9016129 2.7483871 4.39354839 1.43387097] [5.006 3.428 1.462 0.246 ] [6.85 3.07368421 5.74210526 2.07105263]] prediction from Kmeans: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 2 2 2 2 0 2 2 2 2 2 2 0 0 2 2 2 2 0 2 0 2 0 2 2 0 0 2 2 2 2 2 0 2 2 2 2 0 2 2 2 0 2 2 2 0 2 2 0] labels: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2]
## Evaluation of trained model
metrics.accuracy_score(kmeans.labels_, y)
0.24
Human Neurons : Neural Network – 100 billions
Artificial Neurons : Artificial Neural Networks
Machine Learning – To learn parameters
Cost function of ANN
## Import keras and sklearn ML libraries
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam

## DNN Architecture Design for above
## Input Layer : 3 features
## Hiden Layer 1 : 4 Neurons
## Hiden Layer 2 : 4 Neurons
## Output Layer : 0/1
model_dnn = Sequential()
model_dnn.add(Dense(4, input_dim=3, activation='relu'))
model_dnn.add(Dense(4, activation='relu'))
model_dnn.add(Dense(1, activation='sigmoid'))
model_dnn.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense (Dense) (None, 4) 16 _________________________________________________________________ dense_1 (Dense) (None, 4) 20 _________________________________________________________________ dense_2 (Dense) (None, 1) 5 ================================================================= Total params: 41 Trainable params: 41 Non-trainable params: 0 _________________________________________________________________




Plug the parameters
Z = 13.5
For Z = - 0.4
the average of the squared differences between the predicted and actual value
the sum of absolute differences between our target and predicted variables
model.compile(loss='mse', optimizer=Adam(), metrics=['accuracy' ])
For Classification target variables
Optimizer types
Python codes examples for optimizer
###
print(X_train_binary.shape)
print(y_train_binary.shape)
print(y_train_binary)
(80, 4) (80,) [0 0 1 0 0 0 1 0 0 1 1 0 1 1 1 0 0 0 0 0 0 0 1 1 0 0 1 1 1 1 1 1 1 1 0 1 1 0 1 1 1 1 0 1 1 0 1 0 0 0 1 1 0 1 0 0 1 1 1 0 1 0 0 1 1 0 0 1 0 1 0 1 0 0 0 0 1 0 1 1]
## DNN Architecture Design
model_dnn2 = Sequential()
model_dnn2.add(Dense(4, input_dim=4, activation='relu'))
model_dnn2.add(Dense(2, activation='relu'))
model_dnn2.add(Dense(1, activation='sigmoid'))
model_dnn2.summary()
Model: "sequential_1" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_3 (Dense) (None, 4) 20 _________________________________________________________________ dense_4 (Dense) (None, 2) 10 _________________________________________________________________ dense_5 (Dense) (None, 1) 3 ================================================================= Total params: 33 Trainable params: 33 Non-trainable params: 0 _________________________________________________________________
## Compile
model_dnn2.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])
## Train
#model_dnn2.fit(X_train_binary, y_train_binary, epochs=100, batch_size=40)
model_dnn2.fit(X_train_binary, y_train_binary, epochs=10)
Epoch 1/10 3/3 [==============================] - 8s 1ms/step - loss: 0.6930 - accuracy: 0.4320 Epoch 2/10 3/3 [==============================] - 0s 997us/step - loss: 0.6928 - accuracy: 0.5320 Epoch 3/10 3/3 [==============================] - 0s 997us/step - loss: 0.6929 - accuracy: 0.5125 Epoch 4/10 3/3 [==============================] - 0s 499us/step - loss: 0.6928 - accuracy: 0.5102 Epoch 5/10 3/3 [==============================] - 0s 499us/step - loss: 0.6926 - accuracy: 0.4789 Epoch 6/10 3/3 [==============================] - 0s 499us/step - loss: 0.6925 - accuracy: 0.5180 Epoch 7/10 3/3 [==============================] - 0s 499us/step - loss: 0.6927 - accuracy: 0.5437 Epoch 8/10 3/3 [==============================] - 0s 498us/step - loss: 0.6924 - accuracy: 0.5164 Epoch 9/10 3/3 [==============================] - 0s 499us/step - loss: 0.6925 - accuracy: 0.5477 Epoch 10/10 3/3 [==============================] - 0s 499us/step - loss: 0.6927 - accuracy: 0.5086
<keras.callbacks.History at 0x1ef52a17c40>
## Prediction
#dnn_pred = model_dnn2.predict(X_test_binary)
dnn_pred2 = model_dnn2.predict_classes(X_test_binary)
#print the results of the value of centroids
#print("prediction from Kmeans: ", dnn_pred.flatten())
print("prediction from Deep Neural Network (classes): ", dnn_pred2.flatten())
print("labels: ", y_test_binary)
C:\Users\kevin\anaconda3\envs\python-training\lib\site-packages\keras\engine\sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`, if your model does multi-class classification (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype("int32")`, if your model does binary classification (e.g. if it uses a `sigmoid` last-layer activation).
warnings.warn('`model.predict_classes()` is deprecated and '
prediction from Deep Neural Network (classes): [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] labels: [1 1 0 0 1 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0]
## Evaluation - loss function and accuracy0
model_dnn2.evaluate(X_test_binary, y_test_binary)
1/1 [==============================] - 0s 299ms/step - loss: 0.6934 - accuracy: 0.4500
[0.6934430003166199, 0.44999998807907104]
## Weight Parameters of DNN
weights = []
for layer in model_dnn2.layers:
weight = layer.get_weights()
weights.append(weight)
print("1st layer in model: ", len(weights[0]))
print(" weights: ", weights[0][0].shape)
print(" bias : ", weights[0][1].shape)
print("2nd layer in model: ", len(weights[1]))
print(" weights: ", weights[1][0].shape)
print(" bias : ", weights[1][1].shape)
print("3rd layer in model: ", len(weights[2]))
print(" weights: ", weights[2][0].shape)
print(" bias : ", weights[2][1].shape)
1st layer in model: 2
weights: (4, 4)
bias : (4,)
2nd layer in model: 2
weights: (4, 2)
bias : (2,)
3rd layer in model: 2
weights: (2, 1)
bias : (1,)
Your comments and questions are valued and encouraged. Please contact at:
Kevin Lee
AVP of Machine Learning and AI Consultant
Kevin.lee@genpact.com
Tweet: @HelloKevinLee
LinkedIn: www.linkedin.com/in/HelloKevinLee/